package com.project.capture5.app;

import com.project.capture5.bean.UserBehavior;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.util.Collector;

import java.util.HashSet;
import java.util.Set;

/**
 * @author Shelly An
 * @create 2020/9/18 16:02
 */
public class UniqueVisitor {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        //1. 从文件读取数据，转换成bean对象
        SingleOutputStreamOperator<UserBehavior> userBehaviorDS = env.readTextFile("Data/UserBehavior.csv")
                .map(new MapFunction<String, UserBehavior>() {
                    @Override
                    public UserBehavior map(String value) throws Exception {
                        String[] datas = value.split(",");

                        return new UserBehavior(Long.valueOf(datas[0]),
                                Long.valueOf(datas[1]),
                                Integer.valueOf(datas[2]),
                                datas[3],
                                Long.valueOf(datas[4]));
                    }
                });


        //2. 参考WordCount思路，实现uv的统计,对userId进行去重统计,存在一个set中,set.size()==uv

        //2.1 过滤出pv行为
        SingleOutputStreamOperator<UserBehavior> userBehaviorFilter = userBehaviorDS.filter(
                (FilterFunction<UserBehavior>) value -> "pv".equals(value.getBehavior()));

        // 转成二元组，第一个uv是为了分组，分组才能调用sum或process等方法
        //第二个userId是为了存到set里，只需要userId，其他数据丢弃，数据量减少一些
        SingleOutputStreamOperator<Tuple2<String, Long>> uvTuple2 = userBehaviorFilter.map(new MapFunction<UserBehavior, Tuple2<String, Long>>() {
            @Override
            public Tuple2<String, Long> map(UserBehavior value) throws Exception {
                return Tuple2.of("uv", value.getUserId());
            }
        });

        KeyedStream<Tuple2<String, Long>, String> uvKS = uvTuple2.keyBy(data->data.f0);

        SingleOutputStreamOperator<Integer> resultDS = uvKS.process(new KeyedProcessFunction<String, Tuple2<String, Long>, Integer>() {
            //userId放到这里，用来去重
            private Set<Long> uvSet = new HashSet<>();

            /**
             * 来一条处理一条
             * @param value
             * @param ctx
             * @param out
             * @throws Exception
             */
            @Override
            public void processElement(Tuple2<String, Long> value, Context ctx, Collector<Integer> out) throws Exception {
                //来一条数据，就把userId存到Set中
                uvSet.add(value.f1);
                //通过采集器，往下游发送uv值
                out.collect(uvSet.size());
            }
        });

        //215662
        resultDS.print("uv");


        env.execute();
    }
}
